#!/usr/bin/env python
# coding=utf-8
#----------------------------------------------------------------------------
# 文件名：      desc.py
# 说明：       获取股票的行业,涉及概念,地域,产品类型,产品名称,和经营范围
# 作者：       fern
# 创建时间：    2015-4-21
# 版权：       (c) 2015 by fern
# 许可证：      apache license
#----------------------------------------------------------------------------
import mysqlDB
import hashlib,re,sys
from bs4 import BeautifulSoup
from  datetime  import  *
import gzip,urllib2,random,time
import urllib
import StringIO
import json
import socket
import traceback
socket.setdefaulttimeout(10)
reload(sys)
sys.setdefaultencoding('utf-8')

class Desc:
    def __init__(self):
        self.db =mysqlDB.Mydb(host='localhost', dataBase='stock', user='root', passwd='123456')
        self.db.dbLoad()

    def desc(self):
        now=datetime.now()
        date=now.strftime('%Y-%m-%d')
        time=now.strftime('%H:%M:%S')
        #http://stockpage.10jqka.com.cn/601390/
        #http://stockpage.10jqka.com.cn/600886/operate/
        sql="SELECT * FROM  `stock` WHERE id <> 1244  and (`tag`<>'%s' or `desc`<>'%s');" % (date,date)
        query=self.db.query(sql)
        for row in query:
            code=''
            code=row['code'][2:8]
            if code == '300033': #同花顺自己的股 id 1244
                continue
            url="http://stockpage.10jqka.com.cn/%s/operate/" % code
            print "ID: %s Code: %s" % (row['id'],row['code'])
            try:
                html=self.get_gzip(url)
                if html==False:
                    continue
                soup = BeautifulSoup(html)
                intro=soup.find(name="ul",attrs={"class":"main_intro_list"}).find_all(name="li")
                for li in intro:
                    tag_type=li.span.string.strip('：')
                    if tag_type==u'产品类型':
                        product=li.p.string.split('、')
                        for p in product:
                            self.addtag(p.strip(),tag_type,row['code'])
                    if tag_type==u'产品名称':
                        product=li.p.find_all(name="a")
                        for p in product:
                            self.addtag(p.string.strip(),tag_type,row['code'])
                    if tag_type==u'经营范围':
                        product=li.p.string.strip()
                        self.adddesc(row['code'],tag_type,product)

                m_table=soup.find(name="div",attrs={"id":"analysis"}).find_all(name="div",attrs={"class":"m_tab_content"})[0].find(name="table",attrs={"class":"m_table"}).tbody.find_all(name="tr")
                for tr in m_table:
                    if tr.th != None:
                        tag_type=tr.th.string
                    tag=tr.find(name="td",attrs={"class":"tl"}).string.strip()
                    self.addtag(tag,tag_type,row['code'])
                self.update_stock(row['code'],'tag',date)

                #抓取涉及概念及地域
                url="http://stockpage.10jqka.com.cn/%s/" % code
                html=self.get_gzip(url)
                if html==False:
                    continue
                soup = BeautifulSoup(html)
                gsjj=soup.find(name="table",attrs={"class":"gsjj-table"}).tbody.find_all(name="tr")
                for tr in gsjj:
                    td=tr.find_all(name="td")
                    name=td[0].string
                    if name == u'涉及概念':
                        vals=td[1].string.split('，')
                        for val in vals:
                            self.addtag(val,name,row['code'])
                    else:
                        value=td[1].string.strip()
                        self.adddesc(row['code'],name,value)
                self.update_stock(row['code'],'desc',date)
            except:
                continue

    def update_stock(self,code,filed,date):
        sql="UPDATE `stock` SET  `%s`='%s' WHERE `code`='%s';" % (filed,date,code)
        self.db.save(sql)

    def addtag(self,tag,type,code):
        sel="SELECT * FROM `tag` WHERE `name` = '%s' AND `cats`='%s';" % (tag,type)
        query=self.db.query(sel)
        dtime=datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        if query:
            tagid=query[0]['id']
        else:
            sql="INSERT INTO `tag`(`name`, `cats`, `time`) VALUES ('%s','%s','%s')" % (tag,type,dtime)
            self.db.save(sql)
            tagid=self.db.last_id()
        sql="INSERT INTO `stock_tag`(`code`, `tagid`, `time`) VALUES ('%s','%s','%s')" % (code,tagid,dtime)
        self.db.save(sql)

    def adddesc(self,code,name,val):
        sql="INSERT INTO `stock_desc`(`code`, `name`, `val`) VALUES ('%s','%s','%s');" % (code,name,val)
        self.db.save(sql)

    def get_gzip(self,url,num=0):
        if num==None:
            num=0
        elif num>3:
            return False
        header = {'Accept-Charset':'GBK,utf-8;q=0.7,*;q=0.3','User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16'}
        request = urllib2.Request(url,headers=header)
        request.add_header('Accept-encoding', 'gzip')
        opener = urllib2.build_opener()
        try:
            f = opener.open(request,timeout=10)
            isGzip = f.headers.get('Content-Encoding')
            if isGzip :
                compresseddata = f.read()
                compressedstream = StringIO.StringIO(compresseddata)
                gzipper = gzip.GzipFile(fileobj=compressedstream)
                data = gzipper.read()
            else:
                data = f.read()
            return data
        except:
            traceback.print_exc()
            if num<3:
                num=num+1
                self.get_gzip(url,num)

    def get_html(self,url,num=0):
        if num==None:
            num=0
        elif num>3:
            return False
        try:
            if url != None:
                page = urllib.urlopen(url)
                html = page.read()
            else:
                html=""
            return html
        except:
            traceback.print_exc()
            if num<3:
                num=num+1
                self.get_html(url,num)

    def close(self):
        self.db.dbClose()

if __name__ == '__main__':
    print "Prepare data. is come soon ..."
    desc=Desc()
    desc.desc()
    desc.close()
    print "OK!!"
